#include<iostream>
#include<string>
#include<vector>
#include<boost/filesystem.hpp>
#include"util.hpp"
#include"log.hpp"



//html source dic
const std::string src_path = "ingestion/input";
//html processed dic
const std::string dst_path = "ingestion/output/stored_data";

typedef struct docInfo{
    std::string title;
    std::string content;
    std::string url;
}docInfo;

//*为输出型参数
bool EnumFile(const std::string& src_path, std::vector<std::string>* file_list);
bool ParseHtml(const std::vector<std::string>& file_list,  std::vector<docInfo>* mid_res);
bool StoreHtml(const std::vector<docInfo>& mid_res, const std::string& dst_path);

int main()
{
    std::vector<std::string> file_list;
    //1. 递归的将某路径下的 文件路径+文件名 保存在vector中
    if(!EnumFile(src_path, &file_list))
    {
        LOG(ERROR, "EnumFile accessing file name failed");
        return 1;
    }
    //2.根据file_list读取其中的内容，提取有效信息
    std::vector<docInfo> mid_res;
    if(!ParseHtml(file_list, &mid_res))
    {
        LOG(ERROR, "ParseHtml parsing .html file failed");
        return 2;
    }
    //3. 将processed的每一条内容写入dst_path (\3分割)
    if(!StoreHtml(mid_res, dst_path))
    {
        LOG(ERROR, "StoreHtml storing processed .html file failed");
        return 3;
    }

    return 0;
}


bool EnumFile(const std::string& src_path, std::vector<std::string>* file_list)
{
    namespace fs = boost::filesystem;
    fs::path root_path(src_path);
    //1. 路径是否存在
    if(!fs::exists(root_path))
    {
        LOG(ERROR, src_path + " not exists");
        return false;
    }

    //定义一个空的迭代器，指示递归结束
    fs::recursive_directory_iterator end;
    for(fs::recursive_directory_iterator iter(root_path); iter!=end; iter++)
    {
        //普通网页 且 .html
        if(!fs::is_regular_file(*iter))
        {
            continue; //跳过这个文件，检查下一个
        }
        if(iter->path().extension() != ".html")
        {
            continue;
        }
        //std::cout<< iter->path().string() << std::endl; //for debug
        file_list->push_back(iter->path().string());
    }
    return true;
}

//2.1 <title>xxxxx<\title>
static bool ParseTitle(const std::string& file_content, std::string* title)
{
    std::size_t begin = file_content.find("<title>");
    if(begin == std::string::npos)
    {
        return false;
    }

    std::size_t end = file_content.find("</title>");
    if(end == std::string::npos)
    {
        return false;
    }

    begin += std::string("<title>").size();
    if(begin > end)
    {
        return false;
    }

    *title = file_content.substr(begin, end-begin);
    return true;
}

//2.2 去标签
static bool ParseContent(const std::string& file_content, std::string* content)
{
    //自动机
    enum status
    {
        LABLE,
        CONTENT 
    };

    enum status s = LABLE;
    for(char c: file_content)
    {
        switch(s)
        {
            case LABLE:
                if(c == '>') s = CONTENT;
                break;
            case CONTENT:
                if(c == '<') s = LABLE;
                else
                {
                    //不想要\n, 因为想用它分隔后面的文本
                    if(c == '\n') c = ' ';
                    content->push_back(c); //提取有效内容
                }
                break;
            default:
                break;
        }
    }
    return true;
}

//2.3 
static bool ParseUrl(const std::string& file_local_path, std::string* url)
{
    std::string url_head = "https://www.boost.org/doc/libs/1_78_0/doc/html";
    std::string url_tail = file_local_path.substr(src_path.size());
    *url = url_head + url_tail;
    return true;
}

//for debug
void showDoc(const docInfo& doc)
{
    std::cout << doc.title << " " << doc.url << std::endl;
    std::cout << doc.content << std::endl;
}

// 调用ParseTitle, ParseContent 和 StoreHtml
bool ParseHtml(const std::vector<std::string>& file_list,  std::vector<docInfo>* mid_res)
{
    for(const std::string& file : file_list)
    {
        //0. 读取文件内容
        std::string file_content;
        if(!util::FindUtil::ReadFile(file, &file_content))
        {
            continue;
        }

        docInfo doc;
        //1. 提取title - <title>xxxxx<\title>
        if(!ParseTitle(file_content,&doc.title))
        {
            LOG(WARNING, "ParseTitle "+ doc.title +" failed");
            continue;
        }
        //2. 提取content - 去标签
        if(!ParseContent(file_content,&doc.content))
        {
            LOG(WARNING, "ParseContent " + doc.content + " failed");
            continue;
        }
        //3. 提取url
        if(!ParseUrl(file,&doc.url))
        {
            LOG(WARNING, "ParseUrl "+ doc.url +" failed");
            continue;
        }
        mid_res->push_back(std::move(doc)); 

        //Debug
        //showDoc(doc);
    }
    return true;
}

// title\3url\3content\3\n ....
// 方便用getline获得全部文档内容
bool StoreHtml(const std::vector<docInfo>& mid_res, const std::string& dst_path)
{
    std::ofstream ofs(dst_path, std::ios::out | std::ios::binary); //2进制方式写入
    if(!ofs.is_open())
    {
        LOG(ERROR, "StoreHtml open " + dst_path + " failed");
        return false;
    }

    // title\3url\3content\n ....格式写入
    for(auto& file_item: mid_res)
    {
        std::string line;
        line = file_item.title + '\3' + file_item.url + '\3' + file_item.content + '\n';
        ofs.write(line.c_str(), line.size());
    }

    ofs.close();
    return true;
}